import csv
from time import sleep
import requests
from lxml import etree
import csvchange
import random
import re
import os
if not os.path.exists('E:\软件开发与管理'):
	os.mkdir('E:\软件开发与管理')
headers = {"User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Mobile Safari/537.36 Edg/111.0.1661.54"}
response1 = requests.get("https://book.douban.com/tag/?view=type&icn=index-sorttags-all", headers=headers)
html1 = etree.HTML(response1.text)
tags = html1.xpath('//table[@class="tagCol"]/tbody/tr/td/a/text()')
with open('E:\软件开发与管理\豆瓣读书.csv', mode='w', encoding='UTF-8', newline='') as f:
	writer = csv.writer(f)
	writer.writerow(["book_name", "book_author", "book_time", "book_publisher", "book_money", "book_rating", "book_tag"])
	for item in tags:
		for i in range(0, 101, 20):
			number = random.random()
			sleep(number)
			url = "https://book.douban.com/tag/" + item + "?start=" + str(i)
			response = requests.get(url, headers=headers)
			html = etree.HTML(response.text)
			book_name = html.xpath('//div[@class="info"]/h2/a/@title')
			rating_num = html.xpath('//span[@class="rating_nums"]/text()')
			book_info = html.xpath('//div[@class="pub"]/text()')
			book_aut = []
			book_publisher = []
			book_time = []
			book_mo = []
			book_rank = []
			for t in rating_num:
				book_rank.append(t)
			for j in book_info:
				data = j.split('/')
				book_mo.append(data[-1])
				try:
					if len(data) == 5:
						book_time.append(data[3])
						book_publisher.append(data[2])
					else:
						book_publisher.append(data[1])
						book_time.append(data[2])
				except:
					pass
				book_aut.append(data[0])
			book_author = []
			for k in book_aut:
				book_author.append(k.strip())
			book_money_ = []
			book_money = []
			for n in book_mo:
				book_money_.append(n.strip())
			for m in book_money_:
				book_money.append(re.sub('[\u4e00-\u9fa5]', '', m))
			book_data = []
			try:
				for q in range(20):
					book_data.append([book_name[q], book_author[q], book_time[q], book_publisher[q], book_money[q], book_rank[q], item])
			except:
				pass
			print(book_data)
			for ite in book_data:
				writer.writerow(ite)

